package com.example.demo.admin.business;/**
 * Created by kete on 2017-11-04.
 */

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.example.demo.admin.gecco.GeccoPipelineFactory;
import com.example.demo.admin.gecco.Test;
import com.example.demo.admin.gecco.entity.KuaiIpProxyInfo;
import com.example.demo.admin.gecco.entity.XichiIpProxyInfo;
import com.example.demo.common.persistence.dao.IpProxyMapper;
import com.example.demo.common.persistence.dao.MemOrgMapper;
import com.example.demo.common.persistence.dao.SysValueDataMapper;
import com.example.demo.common.persistence.model.IpProxy;
import com.example.demo.common.persistence.model.MemOrg;
import com.example.demo.common.persistence.model.SysValueData;
import com.example.demo.core.util.SpringContextHolder;
import com.geccocrawler.gecco.GeccoEngine;
import com.geccocrawler.gecco.downloader.Proxys;
import com.geccocrawler.gecco.request.HttpGetRequest;
import com.geccocrawler.gecco.request.StartRequestList;
import com.geccocrawler.gecco.spring.SpringPipelineFactory;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import org.apache.commons.collections.map.HashedMap;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.weakref.jmx.internal.guava.collect.Maps;

import java.io.File;
import java.io.IOException;
import java.net.*;
import java.nio.charset.Charset;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 作者：kete
 * 创建时间：2017-11-04 13:51
 * 功能描述：
 * 版本：
 */
@Service
public class IpProxyBusiness {

    private static Logger log = LoggerFactory.getLogger(IpProxyBusiness.class);

    @Autowired
    IpProxyMapper ipProxyMapper;

    @Autowired
    SysValueDataMapper sysValueDataMapper;

    @Autowired
    MemOrgMapper memOrgMapper;


    public static  String ipReg = "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3} \\d{1,6}";

    public static  String[] baseUrl = {"http://www.xicidaili.com/nn/","http://www.xicidaili.com/nt/","http://www.xicidaili.com/wt/",
            "http://www.kuaidaili.com/free/inha/","http://www.kuaidaili.com/free/intr/","http://www.kuaidaili.com/free/outtr/"};

    ThreadLocal<List<IpProxy>> threadLocal = new ThreadLocal<List<IpProxy>>();



    public void insertIpProxy(KuaiIpProxyInfo proxyInfo){
        IpProxy ipProxy = new IpProxy();
        BeanUtils.copyProperties(proxyInfo,ipProxy);
        ipProxy.setSurvivalTime(proxyInfo.getVerifyTime());

        ipProxyMapper.insert(ipProxy);
    }

    public void insertIpProxy(XichiIpProxyInfo proxyInfo){
        KuaiIpProxyInfo kuaiIpProxyInfo = new KuaiIpProxyInfo();
        BeanUtils.copyProperties(proxyInfo,kuaiIpProxyInfo);
        insertIpProxy(kuaiIpProxyInfo);
    }


    public boolean checkProxy(XichiIpProxyInfo proxyInfo){
        KuaiIpProxyInfo kuaiIpProxyInfo = new KuaiIpProxyInfo();
        BeanUtils.copyProperties(proxyInfo,kuaiIpProxyInfo);
        return checkProxy(kuaiIpProxyInfo);
    }


    public boolean checkProxy(KuaiIpProxyInfo proxyInfo){

        try {
            Jsoup.connect("http://1212.ip138.com/ic.asp")
                    .timeout(5 * 1000)
                    .proxy(proxyInfo.getProxyIp(),Integer.parseInt(proxyInfo.getProxyPort()))
                    .get();

            if(ipProxyMapper.checkIpProxy(proxyInfo.getProxyIp())>0) return false ;

            log.info(proxyInfo.toString());

            return true;
        } catch (IOException e) {

        }

        return false;

    }


    public void refreshId(){
        List<IpProxy> list = ipProxyMapper.selectList(new EntityWrapper<>());

        int i =1;

        for(IpProxy ipProxy : list){
            SocketAddress addr = new InetSocketAddress(ipProxy.getProxyIp(),
                    Integer.parseInt(ipProxy.getProxyPort()));
            Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);


            try {
                URL realUrl = new URL("http://www.luoli168.com/portal.php?x=104292");
                // 打开和URL之间的连接
                URLConnection conn = realUrl.openConnection(proxy);
                // 设置通用的请求属性
                conn.setRequestProperty("accept", "*/*");
                conn.setRequestProperty("connection", "Keep-Alive");
                conn.setRequestProperty("User-Agent","Mozilla/4.0 (compatible; MSIE 7.0; NT 5.1; GTB5; .NET CLR 2.0.50727; CIBA)");
                // 建立实际的连接
                conn.connect();
                // 获取所有响应头字段
                Map<String, List<String>> map = conn.getHeaderFields();
                // 遍历所有的响应头字段
                for (String key : map.keySet()) {
                    log.info(key + "--->" + map.get(key));
                }
                Thread.sleep(3000);

                log.info("第"+i+"次访问");
                i++;
            } catch (Exception e) {
                e.printStackTrace();
            }


        }

    }

    public void deleteProxy(){
        List<Integer> ids = Lists.newArrayList();

        List<IpProxy> list = ipProxyMapper.selectList(new EntityWrapper<>());
        list.forEach(ipProxy -> {
            if(!checkProxy(ipProxy.getProxyIp(),Integer.parseInt(ipProxy.getProxyPort())))
                ids.add(ipProxy.getId());
        });

        ipProxyMapper.deleteBatchIds(ids);

    }


    private Integer getOrder(String code){

        String order = code.substring(code.length()-2,code.length());

        return  Integer.parseInt(order);

    }

    private SysValueData paseSysDate(Integer parentId,String key,String value,Integer order){
        SysValueData sysValueData = new SysValueData();
        sysValueData.setOrgId(-1);
        sysValueData.setParentId(parentId);
        sysValueData.setValuesetGroupId(80);
        sysValueData.setValuesetKey(key);
        sysValueData.setValuesetOrder(this.getOrder(key));
        sysValueData.setValuesetValue(value);
        sysValueData.setValuesetValueI18nKey("valueset_addr_city"+key);
        sysValueData.setValuesetEnabled("1");
        sysValueData.setCreateTime(new Date());
        sysValueData.setCreateUserId(1);
        sysValueData.setIsDeleted("0");

        return sysValueData;
    }

    public void initCityEn(){
        String json = null;
        try {
            URL url = Resources.getResource("city.txt");
            File file = new File(url.getPath());
            if(file.exists()) {

                json = Files.toString(file, Charset.forName("UTF-8"));

            }
        } catch(IllegalArgumentException ex) {
            log.info("starts.json not found");
        } catch(Exception ioex) {
            ioex.printStackTrace();
        }

        List<String> strList = Arrays.asList(json.split("/r/n"));

        strList.forEach(name->{
            MemOrg org = memOrgMapper.queryOrgByName(name);

            org.setOrgLevelCode(generateOrgLevelCode());

            memOrgMapper.updateById(org);
        });





    }
    public String generateOrgLevelCode() {
        String orgLevelCode = "000000";
        int maxCodeIndex = memOrgMapper.selectMaxOrgLevelCodeSize(8);
        int orgCodeIndex = maxCodeIndex + 1;
        if (orgCodeIndex > 9999999) return "";
        orgLevelCode += orgCodeIndex;
        orgLevelCode = StringUtils.right(orgLevelCode, 7);
        orgLevelCode = "0" + orgLevelCode;
        return orgLevelCode;
    }


    public void initCity(){

        String json = null;
        try {
            URL url = Resources.getResource("city.json");
            File file = new File(url.getPath());
            if(file.exists()) {
                 json = Files.toString(file, Charset.forName("UTF-8"));

            }
        } catch(IllegalArgumentException ex) {
            log.info("starts.json not found");
        } catch(Exception ioex) {
          ioex.printStackTrace();
        }

        if(json != null){
             JSONObject jsonObject = JSON.parseObject(json);

            EntityWrapper entityWrapper = new EntityWrapper<>();
            entityWrapper.eq("valueset_group_id","84").orderBy("valueset_key");
            List<SysValueData> list = sysValueDataMapper.selectList(entityWrapper);

            for(SysValueData sysValueData : list){

                System.out.printf("------------------------------------------------------:"+sysValueData.getValuesetValue()+"\n");

                if("北京市,天津市,上海市,重庆市".indexOf(sysValueData.getValuesetValue())>=0){


                    JSONObject bj = jsonObject.getJSONObject(sysValueData.getValuesetKey());


                    JSONObject array = bj.getJSONObject("child");

                    Set<String> provinces = array.keySet();

                    provinces.forEach(provice->{

                        JSONObject proObj = array.getJSONObject(provice).getJSONObject("child");

                        Set<String> keys = proObj.keySet();

                        Integer[] index = new Integer[]{1};

                        keys.forEach(key->{

                            String zzObj = proObj.getString(key);

                            SysValueData bean =this.paseSysDate(sysValueData.getId(),key,zzObj,index[0]);

                            index[0] = index[0]++;

                            System.out.println(bean.toString());

                            sysValueDataMapper.insert(bean);


                        });

                    });



                }else if("香港特别行政区,澳门特别行政区,台湾省".indexOf(sysValueData.getValuesetValue())>=0){
                    log.info("啥都不干");
                }else{

                    JSONObject qt = jsonObject.getJSONObject(sysValueData.getValuesetKey());

                    JSONObject array = qt.getJSONObject("child");

                    Set<String> keys = array.keySet();

                    Integer[] index = new Integer[]{1};

                    keys.forEach(key->{

                        JSONObject zzObj = array.getJSONObject(key);


                        SysValueData bean = this.paseSysDate(sysValueData.getId(),key,zzObj.getString("name"),index[0]);

                        index[0] = index[0]++;

                        System.out.println(bean.toString());

                        sysValueDataMapper.insert(bean);

                    });

                }
            }
        }

    }

    public void geccocProxy(){

        EntityWrapper entityWrapper = new EntityWrapper<>();

        List<IpProxy> list = ipProxyMapper.selectList(entityWrapper);

        log.info("list size:"+list.size());

//        list.forEach(ipProxy -> {
//            Proxys.addProxy(ipProxy.getProxyIp(),Integer.parseInt(ipProxy.getProxyPort()));
//        });


        GeccoPipelineFactory springPipelineFactory = SpringContextHolder.getBean("geccoPipelineFactory");


        GeccoEngine geccoEngine = GeccoEngine.create();

        geccoEngine.pipelineFactory(springPipelineFactory)
                .classpath("com.example.demo.admin.gecco.crawler")
                .loop(false)
                .run();

    }

    public void getIpProxy(){

        Pattern ipPtn = Pattern.compile(ipReg);

        int pageIndex = 1;

        while (true){
            if(getCurrentProxyNumber()>100) return;


            try {

                for(int baseUrlIndex=0;baseUrlIndex<5;baseUrlIndex++){


                    Document doc = Jsoup.connect(baseUrl[baseUrlIndex] + pageIndex + "/")
                            .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
                            .header("Accept-Encoding", "gzip, deflate, sdch")
                            .header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6")
                            .header("Cache-Control", "max-age=0")
                            .header("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36")
                            .header("Cookie", "Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1462812244; _gat=1; _ga=GA1.2.1061361785.1462812244")
                            .header("Host", "www.kuaidaili.com")
                            .header("Referer", "http://www.kuaidaili.com/free/outha/")
                            .timeout(30 * 1000)
                            .get();

                    Matcher m = ipPtn.matcher(doc.text());

                    while (m.find()) {
                        if (getCurrentProxyNumber() > 100) {
                            return;
                        }
                        String[] strs = m.group().split(" ");

                        if (checkProxy(strs[0], Integer.parseInt(strs[1]))) {
                            log.info("获取到可用代理IP\t" + strs[0] + "\t" + strs[1]);
                            addProxy(strs[0], strs[1]);
                        }
                    }


                }

                pageIndex++;
            } catch (Exception e) {
                e.printStackTrace();
            }

        }

    }

    private static boolean checkProxy(String ip, Integer port) {
        try {
            //http://1212.ip138.com/ic.asp 可以换成任何比较快的网页
            Jsoup.connect("http://1212.ip138.com/ic.asp")
                    .timeout(4 * 1000)
                    .proxy(ip, port)
                    .get();
            return true;
        } catch (Exception e) {
            return false;
        }
    }


    private int getCurrentProxyNumber() {
        List<IpProxy> proxyInfos = threadLocal.get();
        if (proxyInfos == null) {
            proxyInfos =  Lists.newArrayList();
            threadLocal.set(proxyInfos);
            return 0;
        }
        else {
            return proxyInfos.size();
        }
    }

    private void addProxy(String ip,String port){
        List<IpProxy> list = threadLocal.get();

        if(list == null){
            list = Lists.newArrayList();

        }

        IpProxy ipProxy = new IpProxy();
        ipProxy.setProxyIp(ip);
        ipProxy.setProxyPort(port);

        list.add(ipProxy);
        ipProxyMapper.insert(ipProxy);
    }
}
